{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n",
      "step 100, entropy loss: 0.453008, l2_loss: 3160.498291, total loss: 0.674243\n",
      "0.86\n",
      "step 200, entropy loss: 0.167903, l2_loss: 2960.678955, total loss: 0.375151\n",
      "0.95\n",
      "step 300, entropy loss: 0.178701, l2_loss: 2793.718994, total loss: 0.374261\n",
      "0.92\n",
      "step 400, entropy loss: 0.130160, l2_loss: 2637.272217, total loss: 0.314769\n",
      "0.98\n",
      "step 500, entropy loss: 0.078123, l2_loss: 2501.279053, total loss: 0.253213\n",
      "0.98\n",
      "step 600, entropy loss: 0.034653, l2_loss: 2371.593262, total loss: 0.200665\n",
      "0.98\n",
      "step 700, entropy loss: 0.042068, l2_loss: 2255.158936, total loss: 0.199930\n",
      "0.97\n",
      "step 800, entropy loss: 0.038535, l2_loss: 2144.911865, total loss: 0.188679\n",
      "1.0\n",
      "step 900, entropy loss: 0.054823, l2_loss: 2050.129395, total loss: 0.198332\n",
      "0.98\n",
      "step 1000, entropy loss: 0.046026, l2_loss: 1963.436646, total loss: 0.183467\n",
      "0.98\n"
     ]
    },
    {
     "ename": "ResourceExhaustedError",
     "evalue": "OOM when allocating tensor with shape[10000,28,28,64] and type float on /job:localhost/replica:0/task:0/device:CPU:0 by allocator cpu\n\t [[Node: conv1_7/Conv/Conv2D = Conv2D[T=DT_FLOAT, data_format=\"NHWC\", dilations=[1, 1, 1, 1], padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](reshape_7/Reshape, Conv_28/weights/read)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\nCaused by op 'conv1_7/Conv/Conv2D', defined at:\n  File \"C:\\ProgramData\\Anaconda3\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 478, in start\n    self.io_loop.start()\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\zmq\\eventloop\\ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tornado\\ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2728, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2850, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2910, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-8-ef191470daeb>\", line 32, in <module>\n    weights_initializer=tf.truncated_normal_initializer(0.0, 0.1))\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\framework\\python\\ops\\arg_scope.py\", line 183, in func_with_args\n    return func(*args, **current_args)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\layers\\python\\layers\\layers.py\", line 1154, in convolution2d\n    conv_dims=2)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\framework\\python\\ops\\arg_scope.py\", line 183, in func_with_args\n    return func(*args, **current_args)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\layers\\python\\layers\\layers.py\", line 1057, in convolution\n    outputs = layer.apply(inputs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 774, in apply\n    return self.__call__(inputs, *args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\layers\\base.py\", line 329, in __call__\n    outputs = super(Layer, self).__call__(inputs, *args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 703, in __call__\n    outputs = self.call(inputs, *args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\layers\\convolutional.py\", line 184, in call\n    outputs = self._convolution_op(inputs, self.kernel)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\nn_ops.py\", line 868, in __call__\n    return self.conv_op(inp, filter)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\nn_ops.py\", line 520, in __call__\n    return self.call(inp, filter)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\nn_ops.py\", line 204, in __call__\n    name=self.name)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gen_nn_ops.py\", line 1042, in conv2d\n    data_format=data_format, dilations=dilations, name=name)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3414, in create_op\n    op_def=op_def)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1740, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[10000,28,28,64] and type float on /job:localhost/replica:0/task:0/device:CPU:0 by allocator cpu\n\t [[Node: conv1_7/Conv/Conv2D = Conv2D[T=DT_FLOAT, data_format=\"NHWC\", dilations=[1, 1, 1, 1], padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](reshape_7/Reshape, Conv_28/weights/read)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1321\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1322\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1323\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1306\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1307\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1308\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1408\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1409\u001b[1;33m           run_metadata)\n\u001b[0m\u001b[0;32m   1410\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[10000,28,28,64] and type float on /job:localhost/replica:0/task:0/device:CPU:0 by allocator cpu\n\t [[Node: conv1_7/Conv/Conv2D = Conv2D[T=DT_FLOAT, data_format=\"NHWC\", dilations=[1, 1, 1, 1], padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](reshape_7/Reshape, Conv_28/weights/read)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-ef191470daeb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    108\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m1000\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    109\u001b[0m     print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n\u001b[1;32m--> 110\u001b[1;33m                                     y_: mnist.test.labels, keep_prob:0.5}))\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    898\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    899\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 900\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    901\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    902\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1133\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1134\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1135\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1136\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1137\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1314\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1315\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1316\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1317\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1318\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1333\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1334\u001b[0m           \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1335\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1336\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1337\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[10000,28,28,64] and type float on /job:localhost/replica:0/task:0/device:CPU:0 by allocator cpu\n\t [[Node: conv1_7/Conv/Conv2D = Conv2D[T=DT_FLOAT, data_format=\"NHWC\", dilations=[1, 1, 1, 1], padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](reshape_7/Reshape, Conv_28/weights/read)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\nCaused by op 'conv1_7/Conv/Conv2D', defined at:\n  File \"C:\\ProgramData\\Anaconda3\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 478, in start\n    self.io_loop.start()\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\zmq\\eventloop\\ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tornado\\ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2728, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2850, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2910, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-8-ef191470daeb>\", line 32, in <module>\n    weights_initializer=tf.truncated_normal_initializer(0.0, 0.1))\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\framework\\python\\ops\\arg_scope.py\", line 183, in func_with_args\n    return func(*args, **current_args)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\layers\\python\\layers\\layers.py\", line 1154, in convolution2d\n    conv_dims=2)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\framework\\python\\ops\\arg_scope.py\", line 183, in func_with_args\n    return func(*args, **current_args)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\layers\\python\\layers\\layers.py\", line 1057, in convolution\n    outputs = layer.apply(inputs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 774, in apply\n    return self.__call__(inputs, *args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\layers\\base.py\", line 329, in __call__\n    outputs = super(Layer, self).__call__(inputs, *args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\base_layer.py\", line 703, in __call__\n    outputs = self.call(inputs, *args, **kwargs)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\keras\\layers\\convolutional.py\", line 184, in call\n    outputs = self._convolution_op(inputs, self.kernel)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\nn_ops.py\", line 868, in __call__\n    return self.conv_op(inp, filter)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\nn_ops.py\", line 520, in __call__\n    return self.call(inp, filter)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\nn_ops.py\", line 204, in __call__\n    name=self.name)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\gen_nn_ops.py\", line 1042, in conv2d\n    data_format=data_format, dilations=dilations, name=name)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3414, in create_op\n    op_def=op_def)\n  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1740, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[10000,28,28,64] and type float on /job:localhost/replica:0/task:0/device:CPU:0 by allocator cpu\n\t [[Node: conv1_7/Conv/Conv2D = Conv2D[T=DT_FLOAT, data_format=\"NHWC\", dilations=[1, 1, 1, 1], padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](reshape_7/Reshape, Conv_28/weights/read)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "# Define loss and optimizer\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "with tf.name_scope('conv1'):\n",
    "  h_conv1 = tf.contrib.slim.conv2d(x_image, 64, [10,10],    #输出的卷积核数量、kernel的长宽均增大大一倍\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu,\n",
    "                             weights_initializer=tf.truncated_normal_initializer(0.0, 0.1))  #MSRA初始化，方差设置大一些\n",
    "\n",
    "\n",
    "# Pooling layer - downsamples by 2X.\n",
    "with tf.name_scope('pool1'):\n",
    "  h_pool1 = tf.contrib.slim.max_pool2d(h_conv1, [2,2], stride=2, \n",
    "                         padding='VALID')\n",
    "\n",
    "# Second convolutional layer -- maps 32 feature maps to 64.\n",
    "with tf.name_scope('conv2'):\n",
    "  h_conv2 = tf.contrib.slim.conv2d(h_pool1, 128, [10,10],   #输出的卷积核数量、kernel的长宽均增大大一倍\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu,\n",
    "                             weights_initializer=tf.truncated_normal_initializer(0.0, 0.1))    #MSRA初始化，方差设置大一些\n",
    "\n",
    "# Second pooling layer.\n",
    "with tf.name_scope('pool2'):\n",
    "  h_pool2 = tf.contrib.slim.max_pool2d(h_conv2, [2,2],\n",
    "                        stride=[2, 2], padding='VALID')\n",
    "\n",
    "\n",
    "# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "# is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = tf.contrib.slim.avg_pool2d(h_pool2, h_pool2.shape[1:3],\n",
    "                        stride=[1, 1], padding='VALID')\n",
    "  h_fc1 = tf.contrib.slim.conv2d(h_pool2_flat, 1024, [1,1], activation_fn=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.squeeze(tf.contrib.slim.conv2d(h_fc1_drop, 10, [1,1], activation_fn=None))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 7e-5*l2_loss                #正则化因子先保持不变\n",
    "train_step = tf.train.AdamOptimizer(learning_rate).minimize(total_loss)       #使用性能更好的Adam优化器\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(3000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.001    #减小学习率，避免出现loss越来越大的情况\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从结果的准确率来看，训练400个周期时达到0.98，满足要求；800个周期时达到1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
